{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Predicting drug response using the GDSC model\n",
    "\n",
    "This example shows how to process and predict drug response using the GDSC model based on scRNA-seq data. \n",
    "\n",
    "1. Read normalized bulk gene expression file\n",
    "2. Calculate kernel features\n",
    "3. Predicting drug response using pre-trained GDSC model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-19T01:30:28.813245Z",
     "start_time": "2020-11-19T01:30:27.183948Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ubuntu/anaconda3/envs/cadrres/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:516: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/home/ubuntu/anaconda3/envs/cadrres/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:517: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/home/ubuntu/anaconda3/envs/cadrres/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:518: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/home/ubuntu/anaconda3/envs/cadrres/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:519: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/home/ubuntu/anaconda3/envs/cadrres/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:520: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/home/ubuntu/anaconda3/envs/cadrres/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:525: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n",
      "/home/ubuntu/anaconda3/envs/cadrres/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:541: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/home/ubuntu/anaconda3/envs/cadrres/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:542: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/home/ubuntu/anaconda3/envs/cadrres/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:543: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/home/ubuntu/anaconda3/envs/cadrres/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:544: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/home/ubuntu/anaconda3/envs/cadrres/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:545: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/home/ubuntu/anaconda3/envs/cadrres/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:550: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n"
     ]
    }
   ],
   "source": [
    "import sys, os, pickle\n",
    "import pandas as pd\n",
    "import importlib\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from collections import Counter\n",
    "from scipy import stats\n",
    "\n",
    "import matplotlib as mpl\n",
    "mpl.rcParams['figure.dpi']= 300\n",
    "mpl.rc(\"savefig\", dpi=300)\n",
    "\n",
    "scriptpath = '..'\n",
    "sys.path.append(os.path.abspath(scriptpath))\n",
    "\n",
    "from cadrres import pp, model, evaluation, utility"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Read gene expression file and calculate kernel features"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Indicate input files and output directory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-19T01:30:28.821018Z",
     "start_time": "2020-11-19T01:30:28.817051Z"
    }
   },
   "outputs": [],
   "source": [
    "##########\n",
    "\n",
    "cluster_norm_exp_fname = '../preprocessed_data/HN_patient_specific/log2_fc_cluster_tpm.csv'\n",
    "output_dir = '../result/HN_model/TPM/'\n",
    "\n",
    "# cluster_norm_exp_fname = '../preprocessed_data/HN_patient_specific/log2_fc_patient_tpm.csv'\n",
    "# output_dir = '../result/HN_model/patient_TPM/'\n",
    "\n",
    "# cluster_norm_exp_fname = '../preprocessed_data/HN_patient_specific/log2_fc_tpm.csv'\n",
    "# output_dir = '../result/HN_model/cell_TPM/'\n",
    "\n",
    "##########\n",
    "\n",
    "# cluster_norm_exp_fname = '../preprocessed_data/HN_patient_specific/log2_fc_cluster_tmm.csv'\n",
    "# output_dir = '../result/HN_model/TMM/'\n",
    "\n",
    "# cluster_norm_exp_fname = '../preprocessed_data/HN_patient_specific/log2_fc_patient_tmm.csv'\n",
    "# output_dir = '../result/HN_model/patient_TMM/'\n",
    "\n",
    "# cluster_norm_exp_fname = '../preprocessed_data/HN_patient_specific/log2_fc_cluster_tmm_p95.csv'\n",
    "# output_dir = '../result/HN_model/TMM_p95/'\n",
    "\n",
    "# cluster_norm_exp_fname = '../preprocessed_data/HN_patient_specific/log2_fc_cluster_mat_norm.csv'\n",
    "# output_dir = '../result/HN_model/mat_norm/'\n",
    "\n",
    "# cluster_norm_exp_fname = '../preprocessed_data/HN_patient_specific/log2_fc_cluster_mat_norm_p95.csv'\n",
    "# output_dir = '../result/HN_model/mat_norm_p95/'\n",
    "\n",
    "# cluster_norm_exp_fname = '../preprocessed_data/HN_patient_specific/log2_fc_cluster_mat_norm_log2_p95.csv'\n",
    "# output_dir = '../result/HN_model/mat_norm_log2_p95/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-19T01:30:28.825933Z",
     "start_time": "2020-11-19T01:30:28.823205Z"
    }
   },
   "outputs": [],
   "source": [
    "model_name = 'hn_drug_cw_dw10_100000'\n",
    "# model_name = 'hn_drug_cw_dwsim10_100000'\n",
    "# model_name = 'hn_drug_cw_dw1_100000'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-19T01:30:28.941819Z",
     "start_time": "2020-11-19T01:30:28.827971Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>cluster</th>\n",
       "      <th>A1</th>\n",
       "      <th>A2</th>\n",
       "      <th>B1</th>\n",
       "      <th>B2</th>\n",
       "      <th>C1</th>\n",
       "      <th>C2</th>\n",
       "      <th>D1</th>\n",
       "      <th>D2</th>\n",
       "      <th>E1</th>\n",
       "      <th>E2</th>\n",
       "      <th>...</th>\n",
       "      <th>G1</th>\n",
       "      <th>G2</th>\n",
       "      <th>H1</th>\n",
       "      <th>I1</th>\n",
       "      <th>I2</th>\n",
       "      <th>J1</th>\n",
       "      <th>J2</th>\n",
       "      <th>K1</th>\n",
       "      <th>L</th>\n",
       "      <th>M</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>AAAS</th>\n",
       "      <td>0.336828</td>\n",
       "      <td>0.733255</td>\n",
       "      <td>-0.100211</td>\n",
       "      <td>0.040937</td>\n",
       "      <td>-0.750019</td>\n",
       "      <td>0.441902</td>\n",
       "      <td>0.974277</td>\n",
       "      <td>0.645204</td>\n",
       "      <td>0.244784</td>\n",
       "      <td>-0.114589</td>\n",
       "      <td>...</td>\n",
       "      <td>0.019484</td>\n",
       "      <td>0.136722</td>\n",
       "      <td>0.925250</td>\n",
       "      <td>-0.500852</td>\n",
       "      <td>-0.559264</td>\n",
       "      <td>-0.233178</td>\n",
       "      <td>-2.153421</td>\n",
       "      <td>-0.672916</td>\n",
       "      <td>-1.381729</td>\n",
       "      <td>0.455242</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AAMP</th>\n",
       "      <td>0.473194</td>\n",
       "      <td>0.809530</td>\n",
       "      <td>-0.538478</td>\n",
       "      <td>-0.886829</td>\n",
       "      <td>-1.262913</td>\n",
       "      <td>0.205791</td>\n",
       "      <td>0.197723</td>\n",
       "      <td>-0.083110</td>\n",
       "      <td>0.637455</td>\n",
       "      <td>-0.549449</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.297165</td>\n",
       "      <td>-0.565510</td>\n",
       "      <td>0.710484</td>\n",
       "      <td>-0.937867</td>\n",
       "      <td>-0.961804</td>\n",
       "      <td>-0.854976</td>\n",
       "      <td>-1.168757</td>\n",
       "      <td>-1.749972</td>\n",
       "      <td>-1.927377</td>\n",
       "      <td>0.724136</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AARS</th>\n",
       "      <td>-0.390326</td>\n",
       "      <td>-0.834464</td>\n",
       "      <td>-1.148807</td>\n",
       "      <td>-0.741560</td>\n",
       "      <td>-0.632305</td>\n",
       "      <td>0.353565</td>\n",
       "      <td>1.521685</td>\n",
       "      <td>1.146765</td>\n",
       "      <td>0.464820</td>\n",
       "      <td>-1.386324</td>\n",
       "      <td>...</td>\n",
       "      <td>0.001899</td>\n",
       "      <td>0.299582</td>\n",
       "      <td>0.860722</td>\n",
       "      <td>-0.698699</td>\n",
       "      <td>-1.308104</td>\n",
       "      <td>-1.788221</td>\n",
       "      <td>-1.315991</td>\n",
       "      <td>-0.890714</td>\n",
       "      <td>-0.525582</td>\n",
       "      <td>0.086332</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AARS2</th>\n",
       "      <td>-0.147163</td>\n",
       "      <td>0.771953</td>\n",
       "      <td>-0.073675</td>\n",
       "      <td>0.496356</td>\n",
       "      <td>-0.296694</td>\n",
       "      <td>-0.514839</td>\n",
       "      <td>-0.117732</td>\n",
       "      <td>-0.632430</td>\n",
       "      <td>-0.537640</td>\n",
       "      <td>-1.055336</td>\n",
       "      <td>...</td>\n",
       "      <td>0.332132</td>\n",
       "      <td>-0.194644</td>\n",
       "      <td>-0.078950</td>\n",
       "      <td>-0.259561</td>\n",
       "      <td>0.013180</td>\n",
       "      <td>-0.182999</td>\n",
       "      <td>1.191579</td>\n",
       "      <td>-0.661154</td>\n",
       "      <td>-1.055336</td>\n",
       "      <td>0.897337</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AASDHPPT</th>\n",
       "      <td>0.646827</td>\n",
       "      <td>0.538206</td>\n",
       "      <td>-0.892870</td>\n",
       "      <td>-0.365436</td>\n",
       "      <td>1.541076</td>\n",
       "      <td>0.760270</td>\n",
       "      <td>-0.951734</td>\n",
       "      <td>-0.889768</td>\n",
       "      <td>-1.156785</td>\n",
       "      <td>-0.562848</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.609948</td>\n",
       "      <td>-0.631709</td>\n",
       "      <td>0.407265</td>\n",
       "      <td>-0.445766</td>\n",
       "      <td>-0.539681</td>\n",
       "      <td>-0.372573</td>\n",
       "      <td>-0.907907</td>\n",
       "      <td>-0.100082</td>\n",
       "      <td>-1.228450</td>\n",
       "      <td>0.632283</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 24 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "cluster         A1        A2        B1        B2        C1        C2  \\\n",
       "AAAS      0.336828  0.733255 -0.100211  0.040937 -0.750019  0.441902   \n",
       "AAMP      0.473194  0.809530 -0.538478 -0.886829 -1.262913  0.205791   \n",
       "AARS     -0.390326 -0.834464 -1.148807 -0.741560 -0.632305  0.353565   \n",
       "AARS2    -0.147163  0.771953 -0.073675  0.496356 -0.296694 -0.514839   \n",
       "AASDHPPT  0.646827  0.538206 -0.892870 -0.365436  1.541076  0.760270   \n",
       "\n",
       "cluster         D1        D2        E1        E2  ...        G1        G2  \\\n",
       "AAAS      0.974277  0.645204  0.244784 -0.114589  ...  0.019484  0.136722   \n",
       "AAMP      0.197723 -0.083110  0.637455 -0.549449  ... -0.297165 -0.565510   \n",
       "AARS      1.521685  1.146765  0.464820 -1.386324  ...  0.001899  0.299582   \n",
       "AARS2    -0.117732 -0.632430 -0.537640 -1.055336  ...  0.332132 -0.194644   \n",
       "AASDHPPT -0.951734 -0.889768 -1.156785 -0.562848  ... -0.609948 -0.631709   \n",
       "\n",
       "cluster         H1        I1        I2        J1        J2        K1  \\\n",
       "AAAS      0.925250 -0.500852 -0.559264 -0.233178 -2.153421 -0.672916   \n",
       "AAMP      0.710484 -0.937867 -0.961804 -0.854976 -1.168757 -1.749972   \n",
       "AARS      0.860722 -0.698699 -1.308104 -1.788221 -1.315991 -0.890714   \n",
       "AARS2    -0.078950 -0.259561  0.013180 -0.182999  1.191579 -0.661154   \n",
       "AASDHPPT  0.407265 -0.445766 -0.539681 -0.372573 -0.907907 -0.100082   \n",
       "\n",
       "cluster          L         M  \n",
       "AAAS     -1.381729  0.455242  \n",
       "AAMP     -1.927377  0.724136  \n",
       "AARS     -0.525582  0.086332  \n",
       "AARS2    -1.055336  0.897337  \n",
       "AASDHPPT -1.228450  0.632283  \n",
       "\n",
       "[5 rows x 24 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cluster_norm_exp_df = pd.read_csv(cluster_norm_exp_fname, index_col=0).T\n",
    "cluster_norm_exp_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Read GDSC gene expression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-19T01:30:35.998440Z",
     "start_time": "2020-11-19T01:30:28.943873Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>906826</th>\n",
       "      <th>687983</th>\n",
       "      <th>910927</th>\n",
       "      <th>1240138</th>\n",
       "      <th>1240139</th>\n",
       "      <th>906792</th>\n",
       "      <th>910688</th>\n",
       "      <th>1240135</th>\n",
       "      <th>1290812</th>\n",
       "      <th>907045</th>\n",
       "      <th>...</th>\n",
       "      <th>753584</th>\n",
       "      <th>907044</th>\n",
       "      <th>998184</th>\n",
       "      <th>908145</th>\n",
       "      <th>1659787</th>\n",
       "      <th>1298157</th>\n",
       "      <th>1480372</th>\n",
       "      <th>1298533</th>\n",
       "      <th>930299</th>\n",
       "      <th>905954.1</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GENE</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>TSPAN6</th>\n",
       "      <td>0.941884</td>\n",
       "      <td>0.858532</td>\n",
       "      <td>2.022198</td>\n",
       "      <td>1.107003</td>\n",
       "      <td>1.039128</td>\n",
       "      <td>0.384394</td>\n",
       "      <td>-3.404941</td>\n",
       "      <td>0.271467</td>\n",
       "      <td>-0.747093</td>\n",
       "      <td>-3.234189</td>\n",
       "      <td>...</td>\n",
       "      <td>0.415498</td>\n",
       "      <td>-3.453636</td>\n",
       "      <td>-3.651247</td>\n",
       "      <td>1.683084</td>\n",
       "      <td>0.242039</td>\n",
       "      <td>1.751489</td>\n",
       "      <td>1.732782</td>\n",
       "      <td>1.399116</td>\n",
       "      <td>-3.577806</td>\n",
       "      <td>0.462988</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TNMD</th>\n",
       "      <td>0.108897</td>\n",
       "      <td>-0.077972</td>\n",
       "      <td>-0.212181</td>\n",
       "      <td>-0.037766</td>\n",
       "      <td>0.102051</td>\n",
       "      <td>0.033988</td>\n",
       "      <td>-0.027485</td>\n",
       "      <td>0.019062</td>\n",
       "      <td>-0.168814</td>\n",
       "      <td>0.434495</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.056842</td>\n",
       "      <td>-0.110552</td>\n",
       "      <td>0.120717</td>\n",
       "      <td>-0.003137</td>\n",
       "      <td>-0.233059</td>\n",
       "      <td>-0.216412</td>\n",
       "      <td>0.024201</td>\n",
       "      <td>-0.334520</td>\n",
       "      <td>0.014779</td>\n",
       "      <td>-0.021403</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DPM1</th>\n",
       "      <td>0.343465</td>\n",
       "      <td>1.771254</td>\n",
       "      <td>-0.155355</td>\n",
       "      <td>-0.152617</td>\n",
       "      <td>0.382752</td>\n",
       "      <td>-0.262101</td>\n",
       "      <td>0.228297</td>\n",
       "      <td>0.169844</td>\n",
       "      <td>0.263669</td>\n",
       "      <td>1.534067</td>\n",
       "      <td>...</td>\n",
       "      <td>0.450398</td>\n",
       "      <td>0.406864</td>\n",
       "      <td>0.275874</td>\n",
       "      <td>0.418742</td>\n",
       "      <td>0.382388</td>\n",
       "      <td>1.427654</td>\n",
       "      <td>0.521690</td>\n",
       "      <td>0.756662</td>\n",
       "      <td>-0.162186</td>\n",
       "      <td>0.752131</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SCYL3</th>\n",
       "      <td>-0.632019</td>\n",
       "      <td>-0.179926</td>\n",
       "      <td>-0.290583</td>\n",
       "      <td>-0.183111</td>\n",
       "      <td>0.094688</td>\n",
       "      <td>0.024090</td>\n",
       "      <td>1.721355</td>\n",
       "      <td>-0.531780</td>\n",
       "      <td>-0.398700</td>\n",
       "      <td>1.314070</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.549978</td>\n",
       "      <td>0.377200</td>\n",
       "      <td>0.101711</td>\n",
       "      <td>-0.388691</td>\n",
       "      <td>-0.299252</td>\n",
       "      <td>0.179037</td>\n",
       "      <td>-0.696423</td>\n",
       "      <td>0.196524</td>\n",
       "      <td>0.020015</td>\n",
       "      <td>-0.146319</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C1orf112</th>\n",
       "      <td>-0.313637</td>\n",
       "      <td>0.038166</td>\n",
       "      <td>-0.457698</td>\n",
       "      <td>-0.135904</td>\n",
       "      <td>0.146055</td>\n",
       "      <td>0.120737</td>\n",
       "      <td>-0.682451</td>\n",
       "      <td>-0.425870</td>\n",
       "      <td>-0.341484</td>\n",
       "      <td>-0.123090</td>\n",
       "      <td>...</td>\n",
       "      <td>0.032515</td>\n",
       "      <td>0.253426</td>\n",
       "      <td>0.112265</td>\n",
       "      <td>-0.497331</td>\n",
       "      <td>0.120512</td>\n",
       "      <td>0.690414</td>\n",
       "      <td>0.552870</td>\n",
       "      <td>-0.622959</td>\n",
       "      <td>-0.464122</td>\n",
       "      <td>-0.258524</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 1018 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            906826    687983    910927   1240138   1240139    906792  \\\n",
       "GENE                                                                   \n",
       "TSPAN6    0.941884  0.858532  2.022198  1.107003  1.039128  0.384394   \n",
       "TNMD      0.108897 -0.077972 -0.212181 -0.037766  0.102051  0.033988   \n",
       "DPM1      0.343465  1.771254 -0.155355 -0.152617  0.382752 -0.262101   \n",
       "SCYL3    -0.632019 -0.179926 -0.290583 -0.183111  0.094688  0.024090   \n",
       "C1orf112 -0.313637  0.038166 -0.457698 -0.135904  0.146055  0.120737   \n",
       "\n",
       "            910688   1240135   1290812    907045  ...    753584    907044  \\\n",
       "GENE                                              ...                       \n",
       "TSPAN6   -3.404941  0.271467 -0.747093 -3.234189  ...  0.415498 -3.453636   \n",
       "TNMD     -0.027485  0.019062 -0.168814  0.434495  ... -0.056842 -0.110552   \n",
       "DPM1      0.228297  0.169844  0.263669  1.534067  ...  0.450398  0.406864   \n",
       "SCYL3     1.721355 -0.531780 -0.398700  1.314070  ... -0.549978  0.377200   \n",
       "C1orf112 -0.682451 -0.425870 -0.341484 -0.123090  ...  0.032515  0.253426   \n",
       "\n",
       "            998184    908145   1659787   1298157   1480372   1298533  \\\n",
       "GENE                                                                   \n",
       "TSPAN6   -3.651247  1.683084  0.242039  1.751489  1.732782  1.399116   \n",
       "TNMD      0.120717 -0.003137 -0.233059 -0.216412  0.024201 -0.334520   \n",
       "DPM1      0.275874  0.418742  0.382388  1.427654  0.521690  0.756662   \n",
       "SCYL3     0.101711 -0.388691 -0.299252  0.179037 -0.696423  0.196524   \n",
       "C1orf112  0.112265 -0.497331  0.120512  0.690414  0.552870 -0.622959   \n",
       "\n",
       "            930299  905954.1  \n",
       "GENE                          \n",
       "TSPAN6   -3.577806  0.462988  \n",
       "TNMD      0.014779 -0.021403  \n",
       "DPM1     -0.162186  0.752131  \n",
       "SCYL3     0.020015 -0.146319  \n",
       "C1orf112 -0.464122 -0.258524  \n",
       "\n",
       "[5 rows x 1018 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gdsc_log2_exp_df = pd.read_csv('../data/GDSC/GDSC_exp.tsv', sep='\\t', index_col=0)\n",
    "gdsc_norm_exp_df, gdsc_mean_exp_df = pp.gexp.normalize_log2_mean_fc(gdsc_log2_exp_df)\n",
    "gdsc_norm_exp_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Read an essential gene list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-19T01:30:36.009105Z",
     "start_time": "2020-11-19T01:30:36.000252Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1724"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ess_gene_list = utility.get_gene_list('../data/essential_genes.txt')\n",
    "selected_gene_list = [g for g in ess_gene_list if g in cluster_norm_exp_df.index]\n",
    "len(selected_gene_list)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Calculate kernal features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-19T01:30:38.969292Z",
     "start_time": "2020-11-19T01:30:36.011310Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Calculating kernel features based on 1543 common genes\n",
      "(1724, 24) (17737, 1018)\n"
     ]
    }
   ],
   "source": [
    "cluster_kernel_df = pp.gexp.calculate_kernel_feature(cluster_norm_exp_df, gdsc_norm_exp_df, selected_gene_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-19T01:30:39.024509Z",
     "start_time": "2020-11-19T01:30:38.971844Z"
    }
   },
   "outputs": [],
   "source": [
    "cluster_kernel_df.to_csv(output_dir + \"cluster_kernel_df.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Predicting drug response using pre-trained GDSC model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-19T01:30:39.033795Z",
     "start_time": "2020-11-19T01:30:39.026582Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<module 'cadrres.utility' from '/mnt/volume1/Dropbox/Research/2019_drug_response_heterogeneity/CaDRReS_depository/cadrres/utility.py'>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "importlib.reload(pp)\n",
    "importlib.reload(model)\n",
    "importlib.reload(evaluation)\n",
    "importlib.reload(utility)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-19T01:30:39.381598Z",
     "start_time": "2020-11-19T01:30:39.035516Z"
    }
   },
   "outputs": [],
   "source": [
    "model_dict = pickle.load(open('../result/HN_model/{}_param_dict.pickle.backup'.format(model_name), 'rb'))\n",
    "model_training_output_dict = pickle.load(open('../result/HN_model/{}_output_dict.pickle.backup'.format(model_name), 'rb'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Make a prediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-19T01:30:39.399120Z",
     "start_time": "2020-11-19T01:30:39.384790Z"
    }
   },
   "outputs": [],
   "source": [
    "pred_df, P_df = model.predict_from_model(model_dict, cluster_kernel_df)\n",
    "pred_df.to_csv(os.path.join(output_dir,'pred_{}_model2.csv'.format('gdsc_no_bias_{}').format(model_name)))\n",
    "P_df.to_csv(os.path.join(output_dir,'P_{}_model2.csv'.format('gdsc_no_bias_{}').format(model_name)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-19T01:30:41.709939Z",
     "start_time": "2020-11-19T01:30:41.679512Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>Drug ID</th>\n",
       "      <th>1001</th>\n",
       "      <th>1003</th>\n",
       "      <th>1004</th>\n",
       "      <th>1006</th>\n",
       "      <th>1007</th>\n",
       "      <th>1010</th>\n",
       "      <th>1012</th>\n",
       "      <th>1014</th>\n",
       "      <th>1015</th>\n",
       "      <th>1016</th>\n",
       "      <th>...</th>\n",
       "      <th>299</th>\n",
       "      <th>301</th>\n",
       "      <th>302</th>\n",
       "      <th>303</th>\n",
       "      <th>305</th>\n",
       "      <th>306</th>\n",
       "      <th>308</th>\n",
       "      <th>328</th>\n",
       "      <th>331</th>\n",
       "      <th>346</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>A1</th>\n",
       "      <td>10.398524</td>\n",
       "      <td>-4.984888</td>\n",
       "      <td>-6.042325</td>\n",
       "      <td>0.982309</td>\n",
       "      <td>-6.936394</td>\n",
       "      <td>-0.012527</td>\n",
       "      <td>0.738146</td>\n",
       "      <td>2.654245</td>\n",
       "      <td>3.533897</td>\n",
       "      <td>-1.852266</td>\n",
       "      <td>...</td>\n",
       "      <td>0.565490</td>\n",
       "      <td>2.476670</td>\n",
       "      <td>0.206201</td>\n",
       "      <td>3.186974</td>\n",
       "      <td>3.689912</td>\n",
       "      <td>2.494804</td>\n",
       "      <td>-0.697373</td>\n",
       "      <td>-1.430192</td>\n",
       "      <td>1.614384</td>\n",
       "      <td>-3.958066</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>A2</th>\n",
       "      <td>9.961491</td>\n",
       "      <td>-6.814161</td>\n",
       "      <td>-7.065584</td>\n",
       "      <td>-0.291026</td>\n",
       "      <td>-7.484753</td>\n",
       "      <td>0.024981</td>\n",
       "      <td>0.003014</td>\n",
       "      <td>2.133796</td>\n",
       "      <td>2.968269</td>\n",
       "      <td>-2.287011</td>\n",
       "      <td>...</td>\n",
       "      <td>0.527419</td>\n",
       "      <td>2.163049</td>\n",
       "      <td>0.405491</td>\n",
       "      <td>2.869054</td>\n",
       "      <td>3.224147</td>\n",
       "      <td>2.274660</td>\n",
       "      <td>-1.156717</td>\n",
       "      <td>-2.388423</td>\n",
       "      <td>1.395083</td>\n",
       "      <td>-4.555484</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>B1</th>\n",
       "      <td>11.021523</td>\n",
       "      <td>-4.749574</td>\n",
       "      <td>-5.249366</td>\n",
       "      <td>1.578844</td>\n",
       "      <td>-6.232563</td>\n",
       "      <td>1.137422</td>\n",
       "      <td>1.965151</td>\n",
       "      <td>3.356530</td>\n",
       "      <td>4.349608</td>\n",
       "      <td>-1.453384</td>\n",
       "      <td>...</td>\n",
       "      <td>1.288136</td>\n",
       "      <td>2.588025</td>\n",
       "      <td>0.801405</td>\n",
       "      <td>3.884401</td>\n",
       "      <td>4.243734</td>\n",
       "      <td>2.818478</td>\n",
       "      <td>1.648026</td>\n",
       "      <td>0.313913</td>\n",
       "      <td>1.946526</td>\n",
       "      <td>-1.245332</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>B2</th>\n",
       "      <td>10.646173</td>\n",
       "      <td>-5.557965</td>\n",
       "      <td>-6.364599</td>\n",
       "      <td>0.922382</td>\n",
       "      <td>-7.028233</td>\n",
       "      <td>0.893498</td>\n",
       "      <td>1.615433</td>\n",
       "      <td>3.193993</td>\n",
       "      <td>4.282648</td>\n",
       "      <td>-1.901858</td>\n",
       "      <td>...</td>\n",
       "      <td>0.927751</td>\n",
       "      <td>2.224915</td>\n",
       "      <td>0.364318</td>\n",
       "      <td>3.613059</td>\n",
       "      <td>3.819349</td>\n",
       "      <td>2.404219</td>\n",
       "      <td>1.087065</td>\n",
       "      <td>-0.610679</td>\n",
       "      <td>1.703348</td>\n",
       "      <td>-1.995978</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C1</th>\n",
       "      <td>10.631363</td>\n",
       "      <td>-6.009131</td>\n",
       "      <td>-4.887686</td>\n",
       "      <td>0.626853</td>\n",
       "      <td>-6.369420</td>\n",
       "      <td>0.417760</td>\n",
       "      <td>0.994489</td>\n",
       "      <td>2.138591</td>\n",
       "      <td>2.940815</td>\n",
       "      <td>-1.298257</td>\n",
       "      <td>...</td>\n",
       "      <td>1.860736</td>\n",
       "      <td>2.955682</td>\n",
       "      <td>1.631775</td>\n",
       "      <td>3.751298</td>\n",
       "      <td>4.231162</td>\n",
       "      <td>3.029577</td>\n",
       "      <td>0.037301</td>\n",
       "      <td>-1.023775</td>\n",
       "      <td>2.103131</td>\n",
       "      <td>-2.568724</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C2</th>\n",
       "      <td>10.802532</td>\n",
       "      <td>-4.784965</td>\n",
       "      <td>-4.889357</td>\n",
       "      <td>1.072813</td>\n",
       "      <td>-6.403945</td>\n",
       "      <td>0.107169</td>\n",
       "      <td>1.301386</td>\n",
       "      <td>1.485034</td>\n",
       "      <td>2.833325</td>\n",
       "      <td>-0.879170</td>\n",
       "      <td>...</td>\n",
       "      <td>2.746068</td>\n",
       "      <td>3.192200</td>\n",
       "      <td>2.389507</td>\n",
       "      <td>4.055828</td>\n",
       "      <td>4.336418</td>\n",
       "      <td>3.140493</td>\n",
       "      <td>0.473114</td>\n",
       "      <td>-0.498388</td>\n",
       "      <td>2.831969</td>\n",
       "      <td>-2.106202</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>D1</th>\n",
       "      <td>10.838702</td>\n",
       "      <td>-6.600623</td>\n",
       "      <td>-5.922908</td>\n",
       "      <td>-0.555175</td>\n",
       "      <td>-6.742192</td>\n",
       "      <td>0.846281</td>\n",
       "      <td>0.529191</td>\n",
       "      <td>1.273207</td>\n",
       "      <td>2.231173</td>\n",
       "      <td>-1.557154</td>\n",
       "      <td>...</td>\n",
       "      <td>1.390929</td>\n",
       "      <td>3.063579</td>\n",
       "      <td>1.598017</td>\n",
       "      <td>3.242089</td>\n",
       "      <td>3.852827</td>\n",
       "      <td>2.945125</td>\n",
       "      <td>-1.235541</td>\n",
       "      <td>-2.036298</td>\n",
       "      <td>1.753422</td>\n",
       "      <td>-3.970967</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>D2</th>\n",
       "      <td>10.734875</td>\n",
       "      <td>-6.516898</td>\n",
       "      <td>-6.146479</td>\n",
       "      <td>-0.432917</td>\n",
       "      <td>-7.109557</td>\n",
       "      <td>0.680811</td>\n",
       "      <td>0.601391</td>\n",
       "      <td>1.549414</td>\n",
       "      <td>2.504649</td>\n",
       "      <td>-1.556721</td>\n",
       "      <td>...</td>\n",
       "      <td>0.934664</td>\n",
       "      <td>2.827170</td>\n",
       "      <td>0.951483</td>\n",
       "      <td>3.079927</td>\n",
       "      <td>3.731007</td>\n",
       "      <td>2.690864</td>\n",
       "      <td>-1.728753</td>\n",
       "      <td>-2.726807</td>\n",
       "      <td>1.447842</td>\n",
       "      <td>-4.794363</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>E1</th>\n",
       "      <td>10.570892</td>\n",
       "      <td>-6.884317</td>\n",
       "      <td>-6.937818</td>\n",
       "      <td>-0.966837</td>\n",
       "      <td>-7.755313</td>\n",
       "      <td>0.385784</td>\n",
       "      <td>0.409016</td>\n",
       "      <td>0.796756</td>\n",
       "      <td>2.113789</td>\n",
       "      <td>-1.815461</td>\n",
       "      <td>...</td>\n",
       "      <td>3.191395</td>\n",
       "      <td>3.512492</td>\n",
       "      <td>2.959777</td>\n",
       "      <td>3.920862</td>\n",
       "      <td>4.100335</td>\n",
       "      <td>3.211842</td>\n",
       "      <td>-0.560287</td>\n",
       "      <td>-1.230228</td>\n",
       "      <td>2.807909</td>\n",
       "      <td>-2.703222</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>E2</th>\n",
       "      <td>11.363586</td>\n",
       "      <td>-5.502124</td>\n",
       "      <td>-5.455824</td>\n",
       "      <td>0.253171</td>\n",
       "      <td>-7.403641</td>\n",
       "      <td>0.854778</td>\n",
       "      <td>1.797459</td>\n",
       "      <td>1.257604</td>\n",
       "      <td>2.717944</td>\n",
       "      <td>-1.100992</td>\n",
       "      <td>...</td>\n",
       "      <td>5.772834</td>\n",
       "      <td>4.938277</td>\n",
       "      <td>4.751861</td>\n",
       "      <td>5.533648</td>\n",
       "      <td>5.620082</td>\n",
       "      <td>4.314010</td>\n",
       "      <td>1.184698</td>\n",
       "      <td>1.215393</td>\n",
       "      <td>4.240970</td>\n",
       "      <td>0.358671</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>E3</th>\n",
       "      <td>10.938304</td>\n",
       "      <td>-5.115286</td>\n",
       "      <td>-5.793112</td>\n",
       "      <td>0.588215</td>\n",
       "      <td>-7.032759</td>\n",
       "      <td>0.165687</td>\n",
       "      <td>1.503851</td>\n",
       "      <td>0.848894</td>\n",
       "      <td>2.390273</td>\n",
       "      <td>-1.773371</td>\n",
       "      <td>...</td>\n",
       "      <td>3.036226</td>\n",
       "      <td>3.639978</td>\n",
       "      <td>2.408273</td>\n",
       "      <td>4.403406</td>\n",
       "      <td>4.611050</td>\n",
       "      <td>3.307129</td>\n",
       "      <td>0.671165</td>\n",
       "      <td>0.001023</td>\n",
       "      <td>3.072380</td>\n",
       "      <td>-1.166846</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>F1</th>\n",
       "      <td>11.098785</td>\n",
       "      <td>-7.117527</td>\n",
       "      <td>-6.469940</td>\n",
       "      <td>-0.857574</td>\n",
       "      <td>-7.356102</td>\n",
       "      <td>1.479031</td>\n",
       "      <td>1.199253</td>\n",
       "      <td>1.855947</td>\n",
       "      <td>2.885048</td>\n",
       "      <td>-1.750205</td>\n",
       "      <td>...</td>\n",
       "      <td>2.153209</td>\n",
       "      <td>3.310148</td>\n",
       "      <td>1.999893</td>\n",
       "      <td>3.830136</td>\n",
       "      <td>4.258794</td>\n",
       "      <td>3.104583</td>\n",
       "      <td>-0.149824</td>\n",
       "      <td>-1.168866</td>\n",
       "      <td>2.020800</td>\n",
       "      <td>-2.276948</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>F2</th>\n",
       "      <td>11.463352</td>\n",
       "      <td>-5.852913</td>\n",
       "      <td>-5.265582</td>\n",
       "      <td>0.341153</td>\n",
       "      <td>-6.614839</td>\n",
       "      <td>1.638263</td>\n",
       "      <td>1.834546</td>\n",
       "      <td>1.905653</td>\n",
       "      <td>3.076641</td>\n",
       "      <td>-1.342423</td>\n",
       "      <td>...</td>\n",
       "      <td>4.108223</td>\n",
       "      <td>4.337744</td>\n",
       "      <td>3.443898</td>\n",
       "      <td>4.930240</td>\n",
       "      <td>5.177526</td>\n",
       "      <td>3.918102</td>\n",
       "      <td>1.102709</td>\n",
       "      <td>0.732324</td>\n",
       "      <td>3.274664</td>\n",
       "      <td>-0.224961</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>F3</th>\n",
       "      <td>11.735274</td>\n",
       "      <td>-4.404911</td>\n",
       "      <td>-4.232638</td>\n",
       "      <td>1.345614</td>\n",
       "      <td>-5.310176</td>\n",
       "      <td>1.883102</td>\n",
       "      <td>2.049054</td>\n",
       "      <td>2.731086</td>\n",
       "      <td>3.696949</td>\n",
       "      <td>-1.139711</td>\n",
       "      <td>...</td>\n",
       "      <td>2.531718</td>\n",
       "      <td>4.044702</td>\n",
       "      <td>2.242397</td>\n",
       "      <td>4.448357</td>\n",
       "      <td>4.892552</td>\n",
       "      <td>3.580351</td>\n",
       "      <td>0.901510</td>\n",
       "      <td>0.944763</td>\n",
       "      <td>2.721124</td>\n",
       "      <td>-0.550099</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>G1</th>\n",
       "      <td>10.680516</td>\n",
       "      <td>-6.075951</td>\n",
       "      <td>-5.608131</td>\n",
       "      <td>0.541234</td>\n",
       "      <td>-5.776221</td>\n",
       "      <td>1.302430</td>\n",
       "      <td>0.458796</td>\n",
       "      <td>3.078075</td>\n",
       "      <td>3.399054</td>\n",
       "      <td>-2.084309</td>\n",
       "      <td>...</td>\n",
       "      <td>-1.167922</td>\n",
       "      <td>2.074746</td>\n",
       "      <td>-0.803170</td>\n",
       "      <td>2.414185</td>\n",
       "      <td>3.318371</td>\n",
       "      <td>2.414719</td>\n",
       "      <td>-1.089082</td>\n",
       "      <td>-2.029982</td>\n",
       "      <td>0.395115</td>\n",
       "      <td>-4.906643</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>G2</th>\n",
       "      <td>10.704398</td>\n",
       "      <td>-6.284113</td>\n",
       "      <td>-5.654741</td>\n",
       "      <td>0.115579</td>\n",
       "      <td>-5.970808</td>\n",
       "      <td>1.170492</td>\n",
       "      <td>0.357006</td>\n",
       "      <td>2.297041</td>\n",
       "      <td>2.884502</td>\n",
       "      <td>-1.775684</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.290060</td>\n",
       "      <td>2.418803</td>\n",
       "      <td>0.082566</td>\n",
       "      <td>2.608523</td>\n",
       "      <td>3.375436</td>\n",
       "      <td>2.528537</td>\n",
       "      <td>-1.517694</td>\n",
       "      <td>-2.430944</td>\n",
       "      <td>0.874246</td>\n",
       "      <td>-5.031606</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>H1</th>\n",
       "      <td>10.942833</td>\n",
       "      <td>-4.496451</td>\n",
       "      <td>-4.410276</td>\n",
       "      <td>1.356683</td>\n",
       "      <td>-5.723955</td>\n",
       "      <td>0.247190</td>\n",
       "      <td>1.320766</td>\n",
       "      <td>1.228591</td>\n",
       "      <td>2.509211</td>\n",
       "      <td>-1.209434</td>\n",
       "      <td>...</td>\n",
       "      <td>2.647410</td>\n",
       "      <td>3.576309</td>\n",
       "      <td>2.318976</td>\n",
       "      <td>4.185264</td>\n",
       "      <td>4.491985</td>\n",
       "      <td>3.306451</td>\n",
       "      <td>0.401579</td>\n",
       "      <td>-0.071038</td>\n",
       "      <td>2.967358</td>\n",
       "      <td>-1.646912</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>I1</th>\n",
       "      <td>10.935495</td>\n",
       "      <td>-5.186525</td>\n",
       "      <td>-5.479023</td>\n",
       "      <td>1.273873</td>\n",
       "      <td>-6.598674</td>\n",
       "      <td>0.884427</td>\n",
       "      <td>1.813892</td>\n",
       "      <td>1.979117</td>\n",
       "      <td>3.312676</td>\n",
       "      <td>-1.149087</td>\n",
       "      <td>...</td>\n",
       "      <td>1.152144</td>\n",
       "      <td>2.219583</td>\n",
       "      <td>0.703578</td>\n",
       "      <td>3.509151</td>\n",
       "      <td>4.025896</td>\n",
       "      <td>2.757688</td>\n",
       "      <td>0.947756</td>\n",
       "      <td>-1.285146</td>\n",
       "      <td>1.630458</td>\n",
       "      <td>-3.139641</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>I2</th>\n",
       "      <td>11.332754</td>\n",
       "      <td>-4.386022</td>\n",
       "      <td>-4.921153</td>\n",
       "      <td>1.763705</td>\n",
       "      <td>-6.030852</td>\n",
       "      <td>1.607566</td>\n",
       "      <td>2.050872</td>\n",
       "      <td>3.968558</td>\n",
       "      <td>4.714892</td>\n",
       "      <td>-0.757128</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.390340</td>\n",
       "      <td>1.781665</td>\n",
       "      <td>-0.412108</td>\n",
       "      <td>2.968848</td>\n",
       "      <td>3.912930</td>\n",
       "      <td>2.609815</td>\n",
       "      <td>1.099985</td>\n",
       "      <td>-0.750873</td>\n",
       "      <td>0.666872</td>\n",
       "      <td>-3.461187</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>J1</th>\n",
       "      <td>11.491227</td>\n",
       "      <td>-2.918128</td>\n",
       "      <td>-4.167854</td>\n",
       "      <td>2.522740</td>\n",
       "      <td>-5.378466</td>\n",
       "      <td>1.277405</td>\n",
       "      <td>2.084887</td>\n",
       "      <td>3.108727</td>\n",
       "      <td>4.342564</td>\n",
       "      <td>-0.720091</td>\n",
       "      <td>...</td>\n",
       "      <td>3.973102</td>\n",
       "      <td>4.172037</td>\n",
       "      <td>3.217374</td>\n",
       "      <td>4.975194</td>\n",
       "      <td>5.102332</td>\n",
       "      <td>3.868928</td>\n",
       "      <td>2.072723</td>\n",
       "      <td>2.319877</td>\n",
       "      <td>3.724458</td>\n",
       "      <td>0.259115</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>J2</th>\n",
       "      <td>11.690099</td>\n",
       "      <td>-2.113105</td>\n",
       "      <td>-3.613144</td>\n",
       "      <td>3.237931</td>\n",
       "      <td>-4.926813</td>\n",
       "      <td>1.273595</td>\n",
       "      <td>2.585705</td>\n",
       "      <td>3.094958</td>\n",
       "      <td>4.430778</td>\n",
       "      <td>-0.645442</td>\n",
       "      <td>...</td>\n",
       "      <td>3.411851</td>\n",
       "      <td>4.006978</td>\n",
       "      <td>2.593328</td>\n",
       "      <td>4.953252</td>\n",
       "      <td>5.173830</td>\n",
       "      <td>3.775804</td>\n",
       "      <td>2.441637</td>\n",
       "      <td>2.488317</td>\n",
       "      <td>3.545134</td>\n",
       "      <td>0.440384</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>K1</th>\n",
       "      <td>11.574620</td>\n",
       "      <td>-4.846949</td>\n",
       "      <td>-4.699606</td>\n",
       "      <td>1.119441</td>\n",
       "      <td>-6.172177</td>\n",
       "      <td>1.967038</td>\n",
       "      <td>2.004351</td>\n",
       "      <td>3.755148</td>\n",
       "      <td>4.554336</td>\n",
       "      <td>0.153272</td>\n",
       "      <td>...</td>\n",
       "      <td>0.051219</td>\n",
       "      <td>1.842487</td>\n",
       "      <td>0.333790</td>\n",
       "      <td>2.827999</td>\n",
       "      <td>3.893463</td>\n",
       "      <td>2.707645</td>\n",
       "      <td>0.453652</td>\n",
       "      <td>-1.647179</td>\n",
       "      <td>0.600887</td>\n",
       "      <td>-4.406588</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>L</th>\n",
       "      <td>11.253322</td>\n",
       "      <td>-5.019762</td>\n",
       "      <td>-4.997297</td>\n",
       "      <td>1.158622</td>\n",
       "      <td>-5.902160</td>\n",
       "      <td>1.532919</td>\n",
       "      <td>1.453159</td>\n",
       "      <td>3.348345</td>\n",
       "      <td>3.920958</td>\n",
       "      <td>-1.633050</td>\n",
       "      <td>...</td>\n",
       "      <td>1.353461</td>\n",
       "      <td>3.294049</td>\n",
       "      <td>1.119123</td>\n",
       "      <td>3.821803</td>\n",
       "      <td>4.493831</td>\n",
       "      <td>3.308444</td>\n",
       "      <td>0.612349</td>\n",
       "      <td>0.388145</td>\n",
       "      <td>1.867597</td>\n",
       "      <td>-1.854676</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>M</th>\n",
       "      <td>10.848251</td>\n",
       "      <td>-5.599124</td>\n",
       "      <td>-6.457771</td>\n",
       "      <td>0.326652</td>\n",
       "      <td>-6.081175</td>\n",
       "      <td>1.955663</td>\n",
       "      <td>0.515881</td>\n",
       "      <td>3.601303</td>\n",
       "      <td>4.287474</td>\n",
       "      <td>-1.399306</td>\n",
       "      <td>...</td>\n",
       "      <td>-1.814646</td>\n",
       "      <td>1.640082</td>\n",
       "      <td>-1.342355</td>\n",
       "      <td>1.823583</td>\n",
       "      <td>2.573621</td>\n",
       "      <td>1.729895</td>\n",
       "      <td>-2.444370</td>\n",
       "      <td>-3.751656</td>\n",
       "      <td>-0.023183</td>\n",
       "      <td>-7.107565</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>24 rows × 81 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "Drug ID       1001      1003      1004      1006      1007      1010  \\\n",
       "A1       10.398524 -4.984888 -6.042325  0.982309 -6.936394 -0.012527   \n",
       "A2        9.961491 -6.814161 -7.065584 -0.291026 -7.484753  0.024981   \n",
       "B1       11.021523 -4.749574 -5.249366  1.578844 -6.232563  1.137422   \n",
       "B2       10.646173 -5.557965 -6.364599  0.922382 -7.028233  0.893498   \n",
       "C1       10.631363 -6.009131 -4.887686  0.626853 -6.369420  0.417760   \n",
       "C2       10.802532 -4.784965 -4.889357  1.072813 -6.403945  0.107169   \n",
       "D1       10.838702 -6.600623 -5.922908 -0.555175 -6.742192  0.846281   \n",
       "D2       10.734875 -6.516898 -6.146479 -0.432917 -7.109557  0.680811   \n",
       "E1       10.570892 -6.884317 -6.937818 -0.966837 -7.755313  0.385784   \n",
       "E2       11.363586 -5.502124 -5.455824  0.253171 -7.403641  0.854778   \n",
       "E3       10.938304 -5.115286 -5.793112  0.588215 -7.032759  0.165687   \n",
       "F1       11.098785 -7.117527 -6.469940 -0.857574 -7.356102  1.479031   \n",
       "F2       11.463352 -5.852913 -5.265582  0.341153 -6.614839  1.638263   \n",
       "F3       11.735274 -4.404911 -4.232638  1.345614 -5.310176  1.883102   \n",
       "G1       10.680516 -6.075951 -5.608131  0.541234 -5.776221  1.302430   \n",
       "G2       10.704398 -6.284113 -5.654741  0.115579 -5.970808  1.170492   \n",
       "H1       10.942833 -4.496451 -4.410276  1.356683 -5.723955  0.247190   \n",
       "I1       10.935495 -5.186525 -5.479023  1.273873 -6.598674  0.884427   \n",
       "I2       11.332754 -4.386022 -4.921153  1.763705 -6.030852  1.607566   \n",
       "J1       11.491227 -2.918128 -4.167854  2.522740 -5.378466  1.277405   \n",
       "J2       11.690099 -2.113105 -3.613144  3.237931 -4.926813  1.273595   \n",
       "K1       11.574620 -4.846949 -4.699606  1.119441 -6.172177  1.967038   \n",
       "L        11.253322 -5.019762 -4.997297  1.158622 -5.902160  1.532919   \n",
       "M        10.848251 -5.599124 -6.457771  0.326652 -6.081175  1.955663   \n",
       "\n",
       "Drug ID      1012      1014      1015      1016  ...       299       301  \\\n",
       "A1       0.738146  2.654245  3.533897 -1.852266  ...  0.565490  2.476670   \n",
       "A2       0.003014  2.133796  2.968269 -2.287011  ...  0.527419  2.163049   \n",
       "B1       1.965151  3.356530  4.349608 -1.453384  ...  1.288136  2.588025   \n",
       "B2       1.615433  3.193993  4.282648 -1.901858  ...  0.927751  2.224915   \n",
       "C1       0.994489  2.138591  2.940815 -1.298257  ...  1.860736  2.955682   \n",
       "C2       1.301386  1.485034  2.833325 -0.879170  ...  2.746068  3.192200   \n",
       "D1       0.529191  1.273207  2.231173 -1.557154  ...  1.390929  3.063579   \n",
       "D2       0.601391  1.549414  2.504649 -1.556721  ...  0.934664  2.827170   \n",
       "E1       0.409016  0.796756  2.113789 -1.815461  ...  3.191395  3.512492   \n",
       "E2       1.797459  1.257604  2.717944 -1.100992  ...  5.772834  4.938277   \n",
       "E3       1.503851  0.848894  2.390273 -1.773371  ...  3.036226  3.639978   \n",
       "F1       1.199253  1.855947  2.885048 -1.750205  ...  2.153209  3.310148   \n",
       "F2       1.834546  1.905653  3.076641 -1.342423  ...  4.108223  4.337744   \n",
       "F3       2.049054  2.731086  3.696949 -1.139711  ...  2.531718  4.044702   \n",
       "G1       0.458796  3.078075  3.399054 -2.084309  ... -1.167922  2.074746   \n",
       "G2       0.357006  2.297041  2.884502 -1.775684  ... -0.290060  2.418803   \n",
       "H1       1.320766  1.228591  2.509211 -1.209434  ...  2.647410  3.576309   \n",
       "I1       1.813892  1.979117  3.312676 -1.149087  ...  1.152144  2.219583   \n",
       "I2       2.050872  3.968558  4.714892 -0.757128  ... -0.390340  1.781665   \n",
       "J1       2.084887  3.108727  4.342564 -0.720091  ...  3.973102  4.172037   \n",
       "J2       2.585705  3.094958  4.430778 -0.645442  ...  3.411851  4.006978   \n",
       "K1       2.004351  3.755148  4.554336  0.153272  ...  0.051219  1.842487   \n",
       "L        1.453159  3.348345  3.920958 -1.633050  ...  1.353461  3.294049   \n",
       "M        0.515881  3.601303  4.287474 -1.399306  ... -1.814646  1.640082   \n",
       "\n",
       "Drug ID       302       303       305       306       308       328       331  \\\n",
       "A1       0.206201  3.186974  3.689912  2.494804 -0.697373 -1.430192  1.614384   \n",
       "A2       0.405491  2.869054  3.224147  2.274660 -1.156717 -2.388423  1.395083   \n",
       "B1       0.801405  3.884401  4.243734  2.818478  1.648026  0.313913  1.946526   \n",
       "B2       0.364318  3.613059  3.819349  2.404219  1.087065 -0.610679  1.703348   \n",
       "C1       1.631775  3.751298  4.231162  3.029577  0.037301 -1.023775  2.103131   \n",
       "C2       2.389507  4.055828  4.336418  3.140493  0.473114 -0.498388  2.831969   \n",
       "D1       1.598017  3.242089  3.852827  2.945125 -1.235541 -2.036298  1.753422   \n",
       "D2       0.951483  3.079927  3.731007  2.690864 -1.728753 -2.726807  1.447842   \n",
       "E1       2.959777  3.920862  4.100335  3.211842 -0.560287 -1.230228  2.807909   \n",
       "E2       4.751861  5.533648  5.620082  4.314010  1.184698  1.215393  4.240970   \n",
       "E3       2.408273  4.403406  4.611050  3.307129  0.671165  0.001023  3.072380   \n",
       "F1       1.999893  3.830136  4.258794  3.104583 -0.149824 -1.168866  2.020800   \n",
       "F2       3.443898  4.930240  5.177526  3.918102  1.102709  0.732324  3.274664   \n",
       "F3       2.242397  4.448357  4.892552  3.580351  0.901510  0.944763  2.721124   \n",
       "G1      -0.803170  2.414185  3.318371  2.414719 -1.089082 -2.029982  0.395115   \n",
       "G2       0.082566  2.608523  3.375436  2.528537 -1.517694 -2.430944  0.874246   \n",
       "H1       2.318976  4.185264  4.491985  3.306451  0.401579 -0.071038  2.967358   \n",
       "I1       0.703578  3.509151  4.025896  2.757688  0.947756 -1.285146  1.630458   \n",
       "I2      -0.412108  2.968848  3.912930  2.609815  1.099985 -0.750873  0.666872   \n",
       "J1       3.217374  4.975194  5.102332  3.868928  2.072723  2.319877  3.724458   \n",
       "J2       2.593328  4.953252  5.173830  3.775804  2.441637  2.488317  3.545134   \n",
       "K1       0.333790  2.827999  3.893463  2.707645  0.453652 -1.647179  0.600887   \n",
       "L        1.119123  3.821803  4.493831  3.308444  0.612349  0.388145  1.867597   \n",
       "M       -1.342355  1.823583  2.573621  1.729895 -2.444370 -3.751656 -0.023183   \n",
       "\n",
       "Drug ID       346  \n",
       "A1      -3.958066  \n",
       "A2      -4.555484  \n",
       "B1      -1.245332  \n",
       "B2      -1.995978  \n",
       "C1      -2.568724  \n",
       "C2      -2.106202  \n",
       "D1      -3.970967  \n",
       "D2      -4.794363  \n",
       "E1      -2.703222  \n",
       "E2       0.358671  \n",
       "E3      -1.166846  \n",
       "F1      -2.276948  \n",
       "F2      -0.224961  \n",
       "F3      -0.550099  \n",
       "G1      -4.906643  \n",
       "G2      -5.031606  \n",
       "H1      -1.646912  \n",
       "I1      -3.139641  \n",
       "I2      -3.461187  \n",
       "J1       0.259115  \n",
       "J2       0.440384  \n",
       "K1      -4.406588  \n",
       "L       -1.854676  \n",
       "M       -7.107565  \n",
       "\n",
       "[24 rows x 81 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred_df"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": false,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
